package com.stegniy.webCrawler;

import java.io.*;
import java.net.URL;
import java.util.LinkedHashSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class WebCrawler {

    FileWriter fileWriter;
    File dir;
    int pageCount;

    public WebCrawler() {
        pageCount = 1;
        dir = new File("Downloaded pages");
        dir.mkdirs();
    }

    public String getPage(URL pageUrl) {
        try {
            BufferedReader reader =
                    new BufferedReader(new InputStreamReader(
                            pageUrl.openStream()));

            String line;
            StringBuffer pageBuffer = new StringBuffer();
            while ((line = reader.readLine()) != null) {
                pageBuffer.append(line);
            }
            return pageBuffer.toString();
        } catch (Exception e) {
        }
        return null;
    }

    public void downloadPage(String pageContents) throws IOException {
        File file = new File(dir, "page" + pageCount + ".html");
        file.createNewFile();
        fileWriter = new FileWriter(file);
        fileWriter.write(pageContents);
        fileWriter.close();
        pageCount++;
    }

    public LinkedHashSet<String> getLinks(URL url, String pageContents)
    {
        Pattern p =
                Pattern.compile("<a\\s+href\\s*=\\s*\"?(.*?)[\"|>]",
                        Pattern.CASE_INSENSITIVE);
        Matcher m = p.matcher(pageContents);

        LinkedHashSet<String> linkSet = new LinkedHashSet();
        while (m.find()) {
            String link = m.group(1).trim();

            if (link.length() < 1) {
                continue;
            }

            if (link.charAt(0) == '#') {
                continue;
            }

            if (link.indexOf("mailto:") != -1) {
                continue;
            }

            if (link.toLowerCase().indexOf("javascript") != -1) {
                continue;
            }

            if (link.indexOf("://") == -1) {
                if (link.charAt(0) == '/') {
                    link = "http://" + url.getHost() + link;
                } else {
                    String file = url.getFile();
                    if (file.indexOf('/') == -1) {
                        link = "http://" + url.getHost() + "/" + link;
                    } else {
                        String path =
                                file.substring(0, file.lastIndexOf('/') + 1);
                        link = "http://" + url.getHost() + path + link;
                    }
                }
            }

            int index = link.indexOf('#');
            if (index != -1) {
                link = link.substring(0, index);
            }

            linkSet.add(link);
        }
        return linkSet;
    }
}
